[ { "memory": 1029506304, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 1, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 1, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1032742144, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 1, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 1, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1035977984, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 1, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 1, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1039213824, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 1, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 1, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1039353088, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 1, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 1, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1042588928, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 1, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 1, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1045824768, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 1, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 1, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1046480128, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 1, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1049715968, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 1, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1049855232, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 1, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1053091072, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 1, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1056326912, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 1, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1059562752, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 1, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1062798592, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1066034432, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1069270272, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 1, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1072506112, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1075741952, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1076397312, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1077052672, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 1, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1080288512, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1083524352, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1086760192, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1086899456, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 1, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1087038720, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1090274560, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1090929920, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1091069184, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1094305024, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1097540864, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1100776704, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1104012544, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1107248384, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1110484224, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1113720064, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1113859328, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1113998592, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1117234432, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 1, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1117889792, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1121125632, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1121264896, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1121404160, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1124640000, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1125295360, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1125434624, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 1, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1128670464, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1131906304, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1135142144, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1138377984, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1139033344, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1142269184, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1145505024, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1148740864, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1149396224, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1149535488, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1152771328, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1152910592, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1153049856, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 1, "layer.35.mlp.down_proj": 1 } }, { "memory": 1156285696, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1159521536, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1160176896, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1160832256, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1160971520, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1161110784, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1164346624, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 1, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1164485888, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1167721728, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1170957568, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1171096832, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1171236096, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1171891456, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1175127296, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1178363136, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1179018496, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1182254336, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1182393600, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1185629440, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1188865280, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1192101120, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 1, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1195336960, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 1, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1198572800, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1201808640, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1205044480, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1205699840, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1206355200, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1209591040, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1212826880, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1216062720, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 1, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1219298560, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1222534400, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1225770240, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1229006080, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1229661440, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1230316800, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1230456064, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1230595328, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1230734592, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1233970432, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1237206272, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1237345536, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1238000896, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1238140160, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1238795520, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 1, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1238934784, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1239590144, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1242825984, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1246061824, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1246201088, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1246340352, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1246479616, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1246618880, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1249854720, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1249993984, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1250649344, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1253885184, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1254540544, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1257776384, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1257915648, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1261151488, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1261290752, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 1, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1261946112, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1262085376, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1262740736, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1262880000, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1263535360, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1266771200, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1266910464, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 1, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1267565824, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1267705088, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 1, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1270940928, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1271596288, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1271735552, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1272390912, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1272530176, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1273185536, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1276421376, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1276560640, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1279796480, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 1, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1280451840, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 1, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1280591104, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1281246464, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1281901824, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1282557184, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1282696448, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1283351808, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1283491072, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1284146432, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1284801792, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1285457152, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1285596416, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1285735680, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1288971520, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1289626880, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1289766144, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1290421504, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1290560768, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1291216128, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1291355392, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 1, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1291494656, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1292150016, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1292805376, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1293460736, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1293600000, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1293739264, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 1, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1294394624, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1295049984, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1295705344, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1296360704, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1299596544, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1299735808, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 1, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1299875072, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1300530432, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1301185792, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1301841152, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1305076992, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1305732352, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1305871616, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1306526976, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1309762816, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1310418176, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1311073536, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1314309376, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1317545216, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1318200576, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1318339840, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 1, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1318995200, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1322231040, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1325466880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1326122240, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1326261504, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1329497344, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1332733184, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1335969024, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1336108288, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1336247552, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1339483392, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1339622656, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1339761920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1342997760, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1343137024, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1346372864, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 1, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1346512128, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1346651392, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 1, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1349887232, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1350026496, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1350681856, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1353917696, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1357153536, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1360389376, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1361044736, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1364280576, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1367516416, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1368171776, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1371407616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1371546880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1374782720, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1375438080, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1376093440, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1379329280, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1382565120, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1383220480, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1383875840, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1384015104, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1387250944, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1390486784, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1391142144, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1391797504, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1392452864, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 1, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1392592128, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1395827968, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1399063808, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1402299648, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1405535488, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1408771328, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1408910592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1409565952, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1412801792, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1412941056, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1413080320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1413219584, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1416455424, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1416594688, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1419830528, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1420485888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1420625152, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1423860992, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1427096832, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1427752192, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 1 } }, { "memory": 1430988032, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1431643392, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 1, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1432298752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1435534592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1436189952, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1436329216, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1436984576, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1440220416, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1443456256, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1446692096, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1449927936, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1450583296, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1453819136, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1457054976, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1457194240, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1457333504, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1457472768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1458128128, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1458267392, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1458406656, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1458545920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1458685184, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1461921024, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1465156864, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1465296128, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1468531968, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1469187328, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1469842688, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1473078528, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1476314368, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1476969728, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1477108992, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1477764352, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1477903616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1478042880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1478698240, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1481934080, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1485169920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1488405760, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1491641600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1492296960, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1492952320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1496188160, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1499424000, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1502659840, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1505895680, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1509131520, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1509270784, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 2, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1509410048, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1512645888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1515881728, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1516537088, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1519772928, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1523008768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1526244608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1526899968, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1530135808, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1533371648, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1536607488, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1537262848, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1537918208, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1538573568, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1539228928, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1542464768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1545700608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 2, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1548936448, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1552172288, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1552311552, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1555547392, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1555686656, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1558922496, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 2, "layer.35.mlp.down_proj": 2 } }, { "memory": 1562158336, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 2, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1562813696, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 2, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1562952960, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1566188800, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1569424640, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1569563904, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1572799744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1572939008, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1573594368, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1573733632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1576969472, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1580205312, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1580344576, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1580999936, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 2, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1584235776, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1584891136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1585030400, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 2, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1585169664, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1588405504, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1591641344, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1591780608, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1591919872, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1592575232, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1592714496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1595950336, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1596605696, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1597261056, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1597916416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1598571776, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1601807616, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1601946880, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1602602240, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1603257600, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1606493440, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1609729280, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1609868544, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1610523904, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1610663168, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 2, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1611318528, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1614554368, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1614693632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1617929472, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1621165312, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 2, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1621304576, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1621959936, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1622615296, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 2, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1625851136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1629086976, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1632322816, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1632978176, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1633117440, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1636353280, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1639589120, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1642824960, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1646060800, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1649296640, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 2, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1652532480, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1653187840, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 2, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1653843200, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1657079040, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1657218304, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1657357568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1657496832, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1660732672, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 2, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1663968512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 2, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1667204352, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1670440192, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1671095552, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1671750912, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 2, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1672406272, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1672545536, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1673200896, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 2, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1673340160, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1673995520, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1677231360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 2, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1677886720, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1678025984, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1678165248, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1678820608, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1678959872, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1682195712, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 2, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1682851072, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1683506432, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1686742272, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1689978112, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1690633472, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1693869312, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 2, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1697105152, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 2, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1697244416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1700480256, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1700619520, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1703855360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1703994624, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1704649984, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1707885824, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1708541184, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1708680448, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1709335808, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1709991168, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1713227008, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1716462848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1716602112, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 2, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1719837952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 2, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1723073792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1723729152, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1724384512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1725039872, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1725695232, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1728931072, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1732166912, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1735402752, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1735542016, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1738777856, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1739433216, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1742669056, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1742808320, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1743463680, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1746699520, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1749935360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1750074624, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1750729984, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1750869248, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1751008512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 2, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1751147776, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1751803136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1752458496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1753113856, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1753253120, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1756488960, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1756628224, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1759864064, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1760003328, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1760658688, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1760797952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 2, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1761453312, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 2, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1764689152, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1764828416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1768064256, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1768719616, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1768858880, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1768998144, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1769653504, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1772889344, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1773028608, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1776264448, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1776403712, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1779639552, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1780294912, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1783530752, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1783670016, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1786905856, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1790141696, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1790797056, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1791452416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1794688256, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1794827520, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1794966784, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1798202624, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1798341888, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1798481152, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 2, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1798620416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1801856256, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1802511616, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 2 } }, { "memory": 1805747456, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1808983296, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1812219136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1815454976, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1815594240, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1816249600, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 2, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1816904960, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1820140800, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1823376640, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1826612480, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1827267840, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1830503680, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1833739520, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1836975360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1837114624, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1837769984, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1837909248, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1838048512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1838703872, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1839359232, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1839498496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1839637760, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1842873600, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1846109440, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1849345280, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1852581120, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1853236480, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1856472320, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1859708160, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1859847424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1859986688, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1860125952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1860781312, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1864017152, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1867252992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1867908352, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1871144192, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1871283456, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1871422720, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1874658560, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1877894400, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1881130240, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1884366080, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1884505344, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1887741184, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1887880448, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1891116288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1891771648, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1892427008, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1895662848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1898898688, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1899037952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1902273792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1905509632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1906164992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1909400832, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1910056192, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1910195456, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1913431296, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1916667136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1917322496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1917461760, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1918117120, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1921352960, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1921492224, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1922147584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1922286848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1925522688, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1928758528, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1929413888, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1932649728, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1935885568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1936540928, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1939776768, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1940432128, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1941087488, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1941226752, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1941882112, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1945117952, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1948353792, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1949009152, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1952244992, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1952384256, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1955620096, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1955759360, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1956414720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1957070080, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1960305920, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1960445184, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1963681024, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1966916864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 3, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1970152704, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1973388544, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1974043904, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1977279744, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1977935104, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1978590464, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1981826304, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1982481664, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1985717504, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1988953344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1992189184, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 3, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1992328448, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1992467712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1995703552, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1995842816, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 1999078656, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2002314496, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2002453760, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2005689600, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2008925440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 3, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2012161280, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2015397120, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2016052480, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 3, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2019288320, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 3, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2019427584, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2019566848, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2022802688, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2023458048, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2024113408, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2024768768, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2028004608, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2031240448, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2031379712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2032035072, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2035270912, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2035410176, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2036065536, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 3, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2039301376, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2039956736, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2043192576, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2046428416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2046567680, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2046706944, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2049942784, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2053178624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2053833984, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2057069824, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 3, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2060305664, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2060961024, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2061100288, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 3 } }, { "memory": 2064336128, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2067571968, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2067711232, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 3, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2068366592, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2068505856, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2069161216, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2069816576, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2073052416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2076288256, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2079524096, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2082759936, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2082899200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 3, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2086135040, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2086790400, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2090026240, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 3, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2090165504, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 3, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2093401344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2094056704, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2094195968, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 3, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2097431808, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2100667648, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2103903488, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 3, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2104558848, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2104698112, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2104837376, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2105492736, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2105632000, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2105771264, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2105910528, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2106565888, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 3, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2107221248, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2107360512, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2110596352, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2111251712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2111907072, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2112046336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2112185600, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 3, "layer.35.mlp.down_proj": 4 } }, { "memory": 2115421440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2116076800, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2119312640, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2122548480, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 3, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2122687744, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2123343104, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 3, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2126578944, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2129814784, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2129954048, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2133189888, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2133845248, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2133984512, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2137220352, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2137875712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 3, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2138531072, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 3, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2138670336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2141906176, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2145142016, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 3, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2145797376, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2149033216, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2149688576, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2150343936, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2153579776, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 3, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2153719040, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2156954880, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 3, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2157610240, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 3, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2158265600, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2158920960, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2159576320, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2160231680, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 3, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2163467520, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2164122880, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2167358720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2167497984, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2168153344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2171389184, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2171528448, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2174764288, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 3, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2175419648, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2176075008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2179310848, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 3, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2179450112, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2182685952, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2183341312, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2183996672, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2184135936, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2184275200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2184930560, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 3, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2185069824, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2185209088, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2185864448, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2186003712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2186142976, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2186798336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2187453696, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2187592960, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2190828800, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2194064640, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2194720000, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2197955840, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2198095104, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2198234368, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2198373632, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2201609472, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2201748736, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2204984576, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2205123840, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2205263104, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2208498944, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2211734784, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2214970624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2215625984, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2216281344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2216420608, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2217075968, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2220311808, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2220451072, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2220590336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2220729600, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2223965440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2224104704, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2227340544, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2227995904, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2231231744, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2231371008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2231510272, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2234746112, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2234885376, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2238121216, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2241357056, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2242012416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2242151680, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2242290944, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2242430208, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2243085568, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2246321408, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2249557248, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2252793088, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2256028928, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2259264768, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2262500608, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2265736448, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2266391808, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 4, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2269627648, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2272863488, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2276099328, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 4, "layer.35.mlp.down_proj": 4 } }, { "memory": 2279335168, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2282571008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2285806848, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2289042688, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2292278528, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 4, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2295514368, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2298750208, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2298889472, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2302125312, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2305361152, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2308596992, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2311832832, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2311972096, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2315207936, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2318443776, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 4, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2321679616, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2321818880, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2325054720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2328290560, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2328945920, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 4, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2332181760, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2332321024, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2335556864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 4, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2338792704, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2342028544, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2345264384, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2348500224, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2351736064, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 4 } }, { "memory": 2354971904, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2358207744, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 4, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2361443584, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2364679424, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 4, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2367915264, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2368570624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2371806464, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2372461824, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2375697664, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 4, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2378933504, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2382169344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2382824704, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2386060544, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2386715904, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2386855168, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2390091008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2393326848, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2396562688, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2399798528, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2403034368, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2406270208, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2409506048, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2412741888, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2415977728, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 4, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2419213568, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2422449408, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2425685248, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2428921088, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2432156928, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2435392768, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2438628608, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2441864448, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2445100288, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2448336128, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2451571968, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2451711232, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2451850496, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2452505856, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2453161216, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2456397056, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2459632896, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2460288256, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 4, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2463524096, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2463663360, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2463802624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2464457984, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2467693824, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2467833088, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2471068928, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2474304768, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2474960128, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2475099392, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2475238656, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2478474496, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2479129856, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2482365696, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2483021056, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2483160320, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2483815680, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2484471040, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2487706880, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2490942720, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2494178560, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2497414400, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2498069760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2501305600, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2504541440, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2507777280, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2511013120, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2511152384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2511291648, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2514527488, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2517763328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2520999168, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2521138432, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2524374272, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2527610112, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2527749376, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2528404736, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2529060096, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2529199360, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2529338624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2529477888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2532713728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2533369088, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2533508352, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2536744192, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2536883456, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2537538816, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2538194176, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2541430016, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2542085376, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 4, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2542224640, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 4, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2542880000, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2543535360, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2543674624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2544329984, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2544985344, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 4, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2545124608, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2545779968, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2546435328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2546574592, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2549810432, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2550465792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2553701632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2553840896, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2557076736, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2557732096, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2557871360, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 4, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2558526720, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2561762560, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2564998400, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2568234240, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2568889600, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2572125440, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2572780800, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2576016640, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 4, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2576155904, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2576295168, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2576950528, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2577089792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2580325632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2580980992, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2584216832, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2587452672, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2587591936, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 4, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2587731200, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2588386560, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2589041920, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2589697280, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2592933120, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2593072384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2593727744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2593867008, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2594522368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2594661632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2595316992, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 4, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2595456256, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 4, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2595595520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 4, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2595734784, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2595874048, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2596529408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2597184768, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2597840128, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2597979392, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2601215232, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 4, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2601870592, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2602525952, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2602665216, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2602804480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2603459840, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2603599104, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2604254464, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2604909824, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2605565184, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2606220544, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2606875904, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2607015168, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2607154432, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2607809792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 4, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2607949056, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2611184896, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2611324160, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2611979520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 4, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2612634880, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2613290240, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2613429504, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2613568768, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2613708032, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2613847296, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2617083136, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2617738496, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2617877760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2618017024, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2618672384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2619327744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2622563584, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2623218944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2626454784, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2626594048, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2627249408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2627388672, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2630624512, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2630763776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2633999616, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2634138880, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2637374720, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2637513984, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2637653248, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2637792512, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 4, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2638447872, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 4, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2639103232, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2639242496, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2639897856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2640037120, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2640176384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2640831744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2644067584, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2644722944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2645378304, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2646033664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 4, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2646689024, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2647344384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2650580224, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2650719488, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2650858752, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2654094592, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2654233856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2654373120, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2654512384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 4, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2655167744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2655823104, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2655962368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2656101632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2656240896, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2659476736, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2662712576, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 5, "layer.35.mlp.down_proj": 5 } }, { "memory": 2665948416, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 5 } }, { "memory": 2669184256, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 5 } }, { "memory": 2672420096, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 5 } }, { "memory": 2675655936, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 5 } }, { "memory": 2675795200, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 5, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 5 } }, { "memory": 2675934464, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 5 } }, { "memory": 2679170304, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 5 } }, { "memory": 2682406144, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 5 } }, { "memory": 2685641984, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 5 } }, { "memory": 2688877824, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 5, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 5 } }, { "memory": 2692113664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 5 } }, { "memory": 2695349504, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 5 } }, { "memory": 2698585344, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2698724608, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2701960448, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 5, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2705196288, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2708432128, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2711667968, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2714903808, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 5, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2718139648, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2721375488, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2724611328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2724750592, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 5, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2727986432, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2731222272, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2734458112, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 5, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2737693952, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2740929792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2744165632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 5, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2747401472, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2750637312, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2751292672, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2754528512, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2757764352, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 5, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2761000192, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2764236032, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2767471872, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2768127232, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2771363072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2771502336, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2774738176, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2777974016, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 5, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2781209856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2784445696, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2787681536, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2790917376, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2794153216, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2797389056, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2800624896, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2800764160, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2804000000, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2804139264, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2804794624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2808030464, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2811266304, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 5, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2814502144, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2817737984, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2820973824, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2824209664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2827445504, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2830681344, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2830820608, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2834056448, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2837292288, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2837947648, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2841183488, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2844419328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2847655168, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2850891008, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2854126848, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2857362688, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2860598528, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2860737792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2863973632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2867209472, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2867348736, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 5, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2867488000, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2870723840, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2873959680, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2877195520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 5, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2877334784, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2880570624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2880709888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2883945728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2884601088, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2887836928, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2888492288, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2888631552, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2888770816, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2888910080, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2892145920, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2895381760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2898617600, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2899272960, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2899928320, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2903164160, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2903819520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2903958784, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2907194624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2907333888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2910569728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2913805568, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2914460928, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2915116288, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2918352128, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2919007488, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2922243328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2922382592, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2923037952, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2926273792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2926413056, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 5, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2927068416, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 5, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2927723776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2930959616, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 5, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2934195456, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2934850816, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2935506176, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2935645440, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2936300800, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2936956160, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2937095424, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2937750784, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2937890048, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2941125888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2941265152, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2941404416, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2944640256, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2945295616, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2948531456, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 5, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2948670720, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 5, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2948809984, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2949465344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2952701184, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2952840448, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2952979712, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2956215552, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2956354816, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2957010176, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2960246016, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2963481856, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2964137216, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2964792576, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2968028416, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2968683776, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2971919616, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2972058880, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2972714240, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2975950080, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2976605440, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2977260800, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2977400064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2980635904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2981291264, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2981430528, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2984666368, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2984805632, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2984944896, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2985600256, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2985739520, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 5, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2985878784, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2986018048, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2986157312, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2989393152, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2990048512, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2993284352, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2993423616, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 5, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2994078976, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 5, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2994218240, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2994873600, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2995528960, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2998764800, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2998904064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 2999559424, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3000214784, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3003450624, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3003589888, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3004245248, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3004384512, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3005039872, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3005179136, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3005834496, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3006489856, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3007145216, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3010381056, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3010520320, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3011175680, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3011831040, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3012486400, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3012625664, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3013281024, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3016516864, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3019752704, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3019891968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 5, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3020031232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3020170496, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3020309760, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 5, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3020965120, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3024200960, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3024340224, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3024995584, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3025650944, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 5, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3026306304, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3026445568, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3026584832, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3026724096, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3027379456, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3028034816, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3028174080, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3028829440, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3029484800, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3029624064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3029763328, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3030418688, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3030557952, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3031213312, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3031868672, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3032007936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3032663296, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3032802560, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3033457920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 5, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3034113280, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3034252544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3037488384, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3038143744, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3038283008, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3038938368, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3039593728, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3040249088, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3040904448, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3044140288, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3044279552, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3044934912, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3045590272, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3046245632, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3049481472, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3052717312, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3052856576, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3052995840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3056231680, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 5, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3056887040, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3057026304, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3057165568, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 5, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3057820928, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3057960192, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3058099456, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3058754816, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3058894080, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3059549440, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3062785280, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3062924544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 6 } }, { "memory": 3066160384, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 7 } }, { "memory": 3069396224, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 7 } }, { "memory": 3072632064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 6, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 7 } }, { "memory": 3075867904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 6, "layer.35.mlp.down_proj": 7 } }, { "memory": 3079103744, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 6, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3082339584, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3085575424, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3088811264, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3092047104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3095282944, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3098518784, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3101754624, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 6, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3104990464, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 6, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3105129728, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3108365568, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3111601408, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3114837248, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3118073088, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3121308928, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3124544768, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3124684032, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3127919872, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 6, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3131155712, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3134391552, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3137627392, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3140863232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3144099072, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3147334912, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 6, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3150570752, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 6, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3153806592, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3157042432, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3160278272, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3163514112, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3166749952, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 6, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3169985792, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3173221632, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3176457472, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3179693312, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3182929152, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 6, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3183068416, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3183723776, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3186959616, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3187098880, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3190334720, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 6, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3190473984, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3193709824, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 6, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3196945664, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 6, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3200181504, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3203417344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3204072704, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3204211968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3204351232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3207587072, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3207726336, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3210962176, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3211617536, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3214853376, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3215508736, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3218744576, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3221980416, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3225216256, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3228452096, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 6, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3231687936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3234923776, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3238159616, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3241395456, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3244631296, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3244770560, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3248006400, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3248661760, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 6, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3249317120, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3252552960, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 6, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3252692224, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3255928064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3259163904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3262399744, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3263055104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3266290944, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3269526784, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3270182144, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3270837504, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3274073344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3277309184, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3277964544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3281200384, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3284436224, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3287672064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3290907904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3294143744, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 6, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3294799104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3294938368, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3298174208, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3298313472, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3298452736, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3298592000, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3301827840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3305063680, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3308299520, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3308954880, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3312190720, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3312846080, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3313501440, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3313640704, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3313779968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3314435328, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3317671168, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3317810432, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3317949696, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3318605056, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3319260416, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3322496256, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3325732096, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3328967936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3329623296, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3330278656, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3330934016, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3331073280, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3331728640, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3332384000, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 6, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3332523264, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3333178624, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3333317888, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3333973248, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3334112512, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3337348352, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3337487616, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3338142976, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3338798336, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3339453696, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3342689536, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3345925376, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3349161216, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3349816576, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3349955840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3350095104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3350234368, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3350889728, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3354125568, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 6, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3354780928, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3355436288, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3356091648, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3356747008, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3359982848, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3363218688, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3363357952, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3366593792, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3367249152, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3367388416, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3370624256, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3370763520, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3373999360, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3374138624, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3374793984, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3378029824, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3378169088, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3378824448, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3379479808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3380135168, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3383371008, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3383510272, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3384165632, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3387401472, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3388056832, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3388196096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3391431936, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 6, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3391571200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3391710464, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3391849728, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3392505088, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3393160448, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3393299712, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3393438976, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3396674816, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3397330176, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3397985536, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3398640896, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 6, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3399296256, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3399951616, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3400090880, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3400746240, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3401401600, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3402056960, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3402196224, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3405432064, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3406087424, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3406742784, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3409978624, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3410633984, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 6, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3410773248, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3411428608, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3411567872, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 6, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3412223232, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3412362496, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3413017856, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3413157120, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3416392960, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3416532224, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3417187584, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3417326848, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3420562688, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3421218048, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 6, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3421357312, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3424593152, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3424732416, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3427968256, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3431204096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3431343360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3434579200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3434718464, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3435373824, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3435513088, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3435652352, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3435791616, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3439027456, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3439166720, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3439822080, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3439961344, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3440100608, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3440755968, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3441411328, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3441550592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3441689856, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3442345216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3445581056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3445720320, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3445859584, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3445998848, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3449234688, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3449890048, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3450029312, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3450684672, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3450823936, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3454059776, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3457295616, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 6, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3457950976, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3458606336, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3458745600, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3459400960, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3459540224, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3459679488, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3460334848, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3463570688, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 6, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3464226048, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3464365312, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3464504576, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3464643840, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3465299200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3465438464, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 6, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3466093824, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3466233088, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3466372352, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3469608192, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3472844032, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3476079872, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 7, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3479315712, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 7, "layer.35.mlp.down_proj": 7 } }, { "memory": 3482551552, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3485787392, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3489023232, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3489678592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 7, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3492914432, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 7, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3496150272, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3499386112, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3502621952, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3505857792, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3505997056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3506136320, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3506791680, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 7, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3510027520, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3513263360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3516499200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3519735040, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3522970880, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3526206720, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3529442560, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 7, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3532678400, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3535914240, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 7, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3536053504, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3539289344, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3542525184, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3542664448, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3543319808, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 7, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3546555648, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 7, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3549791488, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3549930752, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3553166592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3553305856, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3556541696, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3559777536, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3563013376, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 7, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3566249216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3566388480, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3569624320, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3572860160, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 7, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3576096000, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 7, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3579331840, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3582567680, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3585803520, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3589039360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3592275200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3595511040, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3596166400, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3599402240, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3602638080, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3605873920, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 7, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3606013184, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3609249024, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3612484864, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3615720704, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3618956544, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3622192384, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3625428224, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3626083584, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3626222848, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3629458688, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3632694528, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3635930368, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3639166208, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3642402048, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3645637888, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3648873728, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 7, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3649012992, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3652248832, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 7, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3652388096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3652527360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3652666624, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3655902464, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3659138304, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3659793664, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3663029504, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3666265344, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3666920704, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3667059968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 7, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3667715328, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3668370688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3669026048, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3669681408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3669820672, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3670476032, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3673711872, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3674367232, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3677603072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3677742336, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3680978176, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3684214016, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3687449856, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 7, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3687589120, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3687728384, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3690964224, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3691619584, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3694855424, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3698091264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3698746624, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3699401984, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3699541248, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3702777088, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3706012928, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3706152192, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3706291456, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3706430720, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3709666560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3712902400, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3713557760, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3713697024, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3714352384, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3717588224, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3717727488, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3718382848, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3721618688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3721757952, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3724993792, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3725649152, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3728884992, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3729540352, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 7, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3729679616, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3730334976, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3730990336, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3731129600, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3731784960, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3735020800, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3735160064, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3735299328, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3738535168, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 7, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3739190528, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3739329792, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3739985152, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3740640512, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3740779776, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3741435136, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3742090496, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3742229760, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3742369024, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3745604864, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3748840704, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3752076544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3752731904, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3755967744, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3756107008, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3756762368, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3756901632, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 7, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3757040896, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3757696256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3758351616, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3758490880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3758630144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3758769408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3762005248, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3765241088, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3768476928, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3768616192, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3771852032, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3775087872, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3775227136, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3775882496, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3779118336, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3782354176, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3785590016, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3786245376, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3786900736, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 7, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3787040000, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3787695360, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3788350720, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3791586560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3792241920, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3795477760, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3798713600, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3801949440, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3802604800, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3802744064, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 7, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3803399424, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3804054784, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3804710144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3805365504, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3805504768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3808740608, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3809395968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3810051328, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3810706688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 7, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3811362048, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3812017408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3812672768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3812812032, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3812951296, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3813090560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3813229824, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3813369088, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3813508352, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3813647616, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3813786880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3813926144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3814581504, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3814720768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3814860032, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3815515392, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3815654656, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3818890496, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3819545856, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3822781696, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3822920960, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3823060224, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3823715584, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3824370944, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3825026304, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3825681664, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3825820928, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3829056768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 7 } }, { "memory": 3832292608, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3832947968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3833603328, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3833742592, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3834397952, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3834537216, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3835192576, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3835847936, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3839083776, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3839223040, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3839878400, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3843114240, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3843253504, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3846489344, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3849725184, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3849864448, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3850003712, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 7, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3850659072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3850798336, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3850937600, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3851592960, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3854828800, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3854968064, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3855107328, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3858343168, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3858482432, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3861718272, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3862373632, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3862512896, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3862652160, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 7, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3863307520, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3863962880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3867198720, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3870434560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 7, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3871089920, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3871745280, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 7, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3872400640, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3873056000, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3876291840, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 8 } }, { "memory": 3879527680, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 8, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 9 } }, { "memory": 3882763520, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 8, "layer.35.mlp.down_proj": 9 } }, { "memory": 3885999360, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3889235200, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 8, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3892471040, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3895706880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3895846144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3895985408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3899221248, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3902457088, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3905692928, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3908928768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3912164608, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 8, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3915400448, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3918636288, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3918775552, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 8, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3922011392, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3925247232, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3928483072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3931718912, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3934954752, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3938190592, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3941426432, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3942081792, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3945317632, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3948553472, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 8, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3951789312, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3955025152, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 8, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3958260992, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3961496832, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3964732672, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3965388032, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3968623872, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3971859712, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3972515072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3975750912, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3978986752, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3979126016, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3979781376, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 8, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3979920640, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3983156480, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3986392320, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3989628160, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3990283520, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 8, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3993519360, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3996755200, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 8, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3996894464, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 3997033728, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4000269568, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4000924928, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4004160768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4004816128, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4004955392, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4005610752, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4008846592, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4008985856, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4009641216, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 8, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4009780480, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4013016320, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4013671680, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4014327040, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4017562880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 8, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4020798720, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4021454080, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4024689920, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4027925760, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4031161600, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4031816960, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4031956224, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 8, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4032095488, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4035331328, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4035470592, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4038706432, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4041942272, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4045178112, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4048413952, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4049069312, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4049208576, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4052444416, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4053099776, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4053239040, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4056474880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4056614144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4057269504, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4060505344, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4063741184, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4066977024, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4067632384, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4068287744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4071523584, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4074759424, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4077995264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4081231104, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4081886464, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4082025728, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4085261568, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4085916928, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4086572288, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4087227648, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4087366912, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4087506176, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4090742016, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4093977856, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4094633216, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4097869056, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4098524416, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4098663680, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4099319040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4099458304, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4099597568, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 8, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4099736832, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4102972672, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4103111936, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4103251200, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 8, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4103906560, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4104561920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4104701184, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4104840448, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4104979712, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 8, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4108215552, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4111451392, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4114687232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4114826496, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4118062336, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4118201600, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4121437440, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4124673280, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4127909120, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4131144960, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4131800320, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 8, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4132455680, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4135691520, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 8, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4136346880, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4139582720, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4142818560, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4142957824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4143097088, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4146332928, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4146988288, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4150224128, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4153459968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4154115328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4154770688, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4155426048, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4155565312, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4156220672, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4156876032, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4157531392, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4157670656, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4157809920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 8, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4158465280, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4158604544, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4159259904, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4159399168, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4159538432, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4159677696, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4162913536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4163052800, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4163192064, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4163331328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4163470592, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4164125952, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4164781312, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4165436672, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 8, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4166092032, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4169327872, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4169983232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4173219072, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4176454912, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4177110272, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4180346112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4183581952, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4183721216, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4183860480, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4183999744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4184655104, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4184794368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4184933632, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 8, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4185072896, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 8, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4185212160, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4188448000, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4191683840, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 8, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4194919680, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4195575040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 8, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4196230400, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4199466240, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4199605504, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4199744768, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4200400128, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4203635968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4204291328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4204946688, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4208182528, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4211418368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4211557632, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4211696896, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4211836160, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4211975424, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4215211264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4218447104, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4219102464, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4219757824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4220413184, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4221068544, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4224304384, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4224443648, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4224582912, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4224722176, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4225377536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4226032896, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4226172160, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4226827520, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 8, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4227482880, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4227622144, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4230857984, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4230997248, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4234233088, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4237468928, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4238124288, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 8, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4238263552, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4238918912, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4239058176, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 8, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4239713536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4240368896, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4240508160, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4240647424, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4241302784, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4241958144, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4245193984, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4245849344, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4249085184, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4249224448, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4249363712, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4250019072, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4253254912, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4253910272, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4257146112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4260381952, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4261037312, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4261176576, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4261831936, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4262487296, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4262626560, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4265862400, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4269098240, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4269237504, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4269376768, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4270032128, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4273267968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4276503808, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4279739648, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4282975488, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4286211328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 9 } }, { "memory": 4289447168, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 9, "layer.35.mlp.down_proj": 10 } }, { "memory": 4292683008, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4295918848, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 9, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4299154688, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4302390528, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4305626368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 9, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4308862208, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4312098048, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4315333888, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4318569728, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4318708992, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4319364352, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4320019712, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4323255552, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4326491392, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4329727232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4330382592, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 9, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4333618432, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4336854272, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 9, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4340090112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4343325952, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4346561792, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4349797632, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4353033472, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 9, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4356269312, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4356408576, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 9, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4359644416, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4362880256, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4366116096, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 9, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4369351936, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4372587776, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4373243136, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4373898496, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4377134336, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4377789696, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4381025536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4384261376, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4387497216, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4390733056, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4393968896, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 9, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4394108160, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4397344000, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4400579840, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4403815680, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 9, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4407051520, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4410287360, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4413523200, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4416759040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4419994880, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 9, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4423230720, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4423886080, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4424025344, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4424680704, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4425336064, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4428571904, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4431807744, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4435043584, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4435182848, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4438418688, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4441654528, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4444890368, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4445029632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4448265472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4451501312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 9, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4454737152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4457972992, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4461208832, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4461864192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4465100032, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4465239296, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4465894656, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4469130496, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4472366336, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4475602176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4478838016, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4482073856, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4482213120, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4482352384, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4485588224, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4488824064, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 9, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4488963328, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4492199168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4492854528, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4493509888, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4496745728, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4497401088, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4498056448, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4498195712, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4501431552, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4504667392, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4507903232, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 9, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4508042496, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4508697856, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 9, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4509353216, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4512589056, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 9, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4512728320, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4515964160, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4516619520, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4516758784, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4517414144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4520649984, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4523885824, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4524025088, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4524680448, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4525335808, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4525475072, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4526130432, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4526269696, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4526925056, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4527064320, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4527203584, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4527858944, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4528514304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4529169664, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4532405504, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4533060864, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4536296704, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4539532544, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4540187904, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4540327168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4543563008, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4544218368, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4544357632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4544496896, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4544636160, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 9, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4544775424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4544914688, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4545053952, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4545709312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4546364672, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4549600512, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4552836352, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4552975616, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4553630976, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4554286336, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4557522176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4560758016, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4561413376, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4561552640, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4562208000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 9, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4562347264, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4565583104, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4566238464, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4569474304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4572710144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4575945984, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4576601344, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4576740608, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4579976448, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4580631808, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4581287168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4581426432, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4584662272, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4584801536, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4585456896, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4586112256, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4586767616, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 9, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4586906880, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4590142720, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4590798080, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4590937344, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4591076608, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4591215872, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4591871232, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4595107072, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4595246336, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4595901696, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4599137536, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 9, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4599276800, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4602512640, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4603168000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4603307264, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4603446528, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4604101888, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4604241152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4604380416, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4607616256, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4607755520, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4608410880, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4608550144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4608689408, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4609344768, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4609484032, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4609623296, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 9, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4610278656, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4610417920, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4610557184, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4611212544, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4611351808, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4611491072, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4614726912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4614866176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4618102016, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 9, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4618757376, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4618896640, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4619552000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4622787840, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4622927104, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4626162944, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4629398784, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4630054144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4633289984, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4633429248, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4633568512, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4636804352, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4640040192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4640179456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4640318720, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4640457984, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4643693824, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4646929664, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4647068928, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 9, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4647724288, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4650960128, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4651099392, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4654335232, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4657571072, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4658226432, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4658881792, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4659537152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4660192512, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4660331776, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4660987136, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4664222976, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4664362240, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4667598080, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4668253440, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 9, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4668908800, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4672144640, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4672283904, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4672939264, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4673078528, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4673217792, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4673357056, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4674012416, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4674667776, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4674807040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4674946304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4678182144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4678321408, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4681557248, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4682212608, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4682351872, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 9, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4683007232, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4683662592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4684317952, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 9, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4684973312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4685112576, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 9, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4685767936, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4686423296, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4689659136, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 10 } }, { "memory": 4692894976, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 10, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 11 } }, { "memory": 4696130816, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 10, "layer.35.mlp.down_proj": 11 } }, { "memory": 4699366656, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4702602496, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4705838336, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4709074176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 10, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4712310016, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4715545856, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4718781696, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 10, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4722017536, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4725253376, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4725908736, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4726048000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4726187264, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4729423104, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4730078464, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4733314304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4736550144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4739785984, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4743021824, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4746257664, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4749493504, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 10, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4752729344, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4755965184, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4759201024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4762436864, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4763092224, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4766328064, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4766983424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4767638784, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4770874624, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4774110464, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 10, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4777346304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4780582144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4780721408, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 10, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4783957248, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4787193088, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4790428928, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4790568192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4793804032, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4794459392, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4797695232, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4800931072, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4804166912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 10, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4807402752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4810638592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4813874432, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4817110272, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4820346112, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 10, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4821001472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4824237312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4827473152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4830708992, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4833944832, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4837180672, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4837836032, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 10, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4841071872, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4841211136, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4841866496, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4842005760, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4842145024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4845380864, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4848616704, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4849272064, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4849411328, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4852647168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4852786432, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4853441792, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4853581056, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 10, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4854236416, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4854891776, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4855031040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4855170304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4858406144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4859061504, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4862297344, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4862952704, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4866188544, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4866327808, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4869563648, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4870219008, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 10, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4873454848, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4876690688, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4876829952, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4877485312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4880721152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4883956992, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4884096256, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4887332096, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4887987456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4888642816, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4888782080, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4892017920, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4892673280, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4893328640, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4893984000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4897219840, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4900455680, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4900594944, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 10, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4901250304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4901389568, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4901528832, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4902184192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4902323456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4902462720, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4905698560, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4905837824, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4905977088, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4909212928, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4909868288, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4913104128, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4913759488, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4913898752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4917134592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4917789952, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 10, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4917929216, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4918068480, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4918207744, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4918863104, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4922098944, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4922238208, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4922377472, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4925613312, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4928849152, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4929504512, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4932740352, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 10, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4932879616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4933534976, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4933674240, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4933813504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4934468864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4935124224, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4935263488, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4938499328, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4939154688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4942390528, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4943045888, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 10, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4943185152, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4943324416, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4943979776, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 10, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4944635136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4945290496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 10, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4945429760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4948665600, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4951901440, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4955137280, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4955276544, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4955415808, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4955555072, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 10, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4955694336, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4956349696, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4956488960, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4956628224, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4956767488, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4957422848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4960658688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4961314048, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4961969408, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 10, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4962108672, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4965344512, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4968580352, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4968719616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4969374976, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4972610816, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4973266176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4973921536, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4974576896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4975232256, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 10, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4975371520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4978607360, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 10, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4978746624, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 10, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4979401984, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4979541248, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4979680512, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4980335872, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4980475136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4981130496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4981269760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4981409024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4984644864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4985300224, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 10, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4985955584, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4986094848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4989330688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4992566528, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4993221888, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4993877248, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 4997113088, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5000348928, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5001004288, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5001143552, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5004379392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5004518656, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5007754496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5010990336, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5011129600, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5011268864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5011408128, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5011547392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5012202752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5012342016, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5015577856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5015717120, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5016372480, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5017027840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5017683200, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5017822464, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 10, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5021058304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5021713664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5021852928, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5022508288, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5022647552, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5023302912, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5026538752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5029774592, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5029913856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5033149696, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5036385536, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5037040896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5037696256, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5040932096, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5044167936, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5044307200, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5044962560, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5045617920, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5046273280, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5049509120, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5050164480, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5053400320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5054055680, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5054711040, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5057946880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 10, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5058602240, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5058741504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5058880768, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5062116608, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5065352448, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5066007808, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5069243648, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5072479488, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5075715328, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5075854592, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5075993856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5079229696, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 10, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5079885056, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5083120896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5083260160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5086496000, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5086635264, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5089871104, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5093106944, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5096342784, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 11 } }, { "memory": 5099578624, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 11, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 12 } }, { "memory": 5102814464, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 12 } }, { "memory": 5106050304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 11, "layer.35.mlp.down_proj": 12 } }, { "memory": 5109286144, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5112521984, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5115757824, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5118993664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5122229504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5125465344, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5128701184, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5131937024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 11, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5135172864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5138408704, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5141644544, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5141783808, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5145019648, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 11, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5148255488, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5151491328, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5154727168, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 11, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5157963008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 11, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5161198848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5164434688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5167670528, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 11, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5170906368, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5174142208, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 11, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5177378048, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5180613888, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5183849728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5187085568, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5190321408, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5193557248, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5196793088, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 11, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5196932352, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 11, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5197071616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5200307456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5203543296, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5206779136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5210014976, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5213250816, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5216486656, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5219722496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 11, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5220377856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5223613696, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5226849536, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5230085376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5230740736, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5231396096, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5232051456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5232706816, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5235942656, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5239178496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5239833856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 11, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5239973120, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5240112384, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5243348224, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5246584064, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5249819904, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5253055744, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 11, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5256291584, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5256430848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5256570112, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5259805952, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5263041792, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5266277632, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5266416896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5269652736, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5270308096, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5273543936, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5276779776, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5280015616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 11, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5283251456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5286487296, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5287142656, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5290378496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5291033856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5294269696, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5294925056, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5298160896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5301396736, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5302052096, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5305287936, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 11, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5305427200, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5308663040, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5309318400, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5309973760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5313209600, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5313348864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 11, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5313488128, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5313627392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5314282752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5317518592, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5320754432, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 11, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5321409792, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5321549056, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5322204416, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5325440256, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5325579520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5326234880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5326374144, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5327029504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5330265344, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5333501184, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5333640448, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5333779712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 11, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5334435072, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5337670912, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5338326272, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5338981632, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5339636992, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5339776256, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 11, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5339915520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5340570880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5340710144, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 11, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5340849408, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5341504768, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5342160128, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5342299392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5345535232, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5348771072, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5348910336, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5349049600, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5349188864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5349328128, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5349983488, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5350122752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5350778112, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5350917376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5351056640, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5354292480, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5354947840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5355087104, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5358322944, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5358978304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5359633664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5359772928, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5360428288, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5363664128, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5366899968, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5367555328, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5368210688, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5368866048, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5369521408, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5372757248, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5372896512, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5373035776, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5373175040, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5373830400, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5373969664, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5377205504, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5377860864, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5381096704, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 11, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5381235968, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5381891328, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5382030592, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5382169856, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5382825216, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5382964480, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5383103744, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5383759104, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5384414464, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5384553728, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5387789568, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5387928832, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5388068096, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5391303936, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5394539776, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5395195136, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5398430976, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5399086336, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5399741696, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 11, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5400397056, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5403632896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5404288256, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5407524096, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5408179456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5408834816, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5408974080, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5409113344, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5409252608, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5412488448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5415724288, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5415863552, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5416518912, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5417174272, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5417829632, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5417968896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5418624256, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5419279616, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 11, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5422515456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5425751296, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5426406656, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5426545920, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5426685184, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5427340544, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5427479808, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5428135168, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5428790528, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5432026368, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5432165632, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5432304896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5432444160, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5432583424, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5432722688, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5433378048, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5434033408, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5437269248, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5440505088, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5440644352, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5443880192, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5444019456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5444158720, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5447394560, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 11, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5448049920, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5448189184, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5448328448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5451564288, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5451703552, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5454939392, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5458175232, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5458830592, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5459485952, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5462721792, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5462861056, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5463516416, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5463655680, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5463794944, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5464450304, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5467686144, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5470921984, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5471061248, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 11, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5471716608, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5474952448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5475091712, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5478327552, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5481563392, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5482218752, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5482358016, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5485593856, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5485733120, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5486388480, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 11, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5487043840, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5487183104, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5487322368, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5490558208, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5490697472, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5491352832, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5494588672, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 11, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5495244032, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5495899392, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5496554752, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5499790592, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5503026432, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5506262272, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5509498112, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 12, "layer.35.mlp.down_proj": 12 } }, { "memory": 5512733952, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5515969792, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5519205632, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 12, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5522441472, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5525677312, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 12, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5528913152, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5532148992, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 12, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5532288256, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5532943616, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5536179456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5536834816, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5540070656, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 12, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5543306496, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5546542336, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5549778176, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5553014016, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5556249856, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5559485696, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5562721536, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5565957376, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5569193216, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5572429056, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5575664896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5575804160, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5576459520, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 12, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5579695360, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5582931200, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5586167040, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5589402880, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 12, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5592638720, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5595874560, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5599110400, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5602346240, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5602485504, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5602624768, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5605860608, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5609096448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5609751808, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5610407168, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5613643008, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5616878848, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5617018112, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5620253952, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5620393216, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5620532480, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5623768320, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 12, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5627004160, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 12 } }, { "memory": 5630240000, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5633475840, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5633615104, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5636850944, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5637506304, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5640742144, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5641397504, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5641536768, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5644772608, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5645427968, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5645567232, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5648803072, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5652038912, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5652694272, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5653349632, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5654004992, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 12, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5654660352, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 12, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5654799616, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5654938880, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5655594240, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5658830080, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 12, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5662065920, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5662721280, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5663376640, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5666612480, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5669848320, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5669987584, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5673223424, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5673362688, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5673501952, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5674157312, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5674296576, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5674435840, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5677671680, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5678327040, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5678466304, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5678605568, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5681841408, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 12, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5682496768, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5685732608, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5686387968, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 12, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5689623808, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5689763072, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5689902336, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5690041600, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5690180864, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 12, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5693416704, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5696652544, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5696791808, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 12, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5700027648, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5703263488, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5703402752, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5703542016, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5704197376, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5704852736, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5705508096, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 12, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5705647360, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5706302720, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5709538560, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5710193920, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5710849280, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5714085120, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5714740480, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5717976320, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 12, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5718631680, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5721867520, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5722522880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5722662144, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5723317504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5723456768, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5723596032, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5726831872, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5727487232, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5728142592, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5728797952, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5729453312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5729592576, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5729731840, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5730387200, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5733623040, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5733762304, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5733901568, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5737137408, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5737792768, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5741028608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 12, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5741683968, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5742339328, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 12, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5742994688, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5746230528, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5746369792, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5747025152, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5750260992, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 12, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5753496832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5753636096, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5754291456, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5754430720, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5754569984, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 12, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5754709248, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5754848512, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5755503872, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5756159232, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5756814592, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5757469952, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5758125312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5758780672, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5762016512, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5762155776, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5762811136, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5762950400, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5763605760, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5766841600, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5767496960, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 12, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5767636224, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5768291584, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 12, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5768946944, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5772182784, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5772322048, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 12, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5772461312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5773116672, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5773255936, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5776491776, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5776631040, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5776770304, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5777425664, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5778081024, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5778736384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5781972224, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5782111488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 12, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5782766848, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5782906112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5786141952, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5786281216, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5786420480, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5787075840, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5787215104, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5790450944, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5791106304, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5794342144, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5794997504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 12, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5795136768, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5798372608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5801608448, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5801747712, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5802403072, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5802542336, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5805778176, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5806433536, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5807088896, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5810324736, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5813560576, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5816796416, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5820032256, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5820687616, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5823923456, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5827159296, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5827814656, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5827953920, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5828093184, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 12, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5828232448, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5828371712, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5831607552, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5832262912, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5832402176, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5832541440, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5833196800, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5836432640, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5836571904, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5836711168, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5836850432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5836989696, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5837645056, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5837784320, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5841020160, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5841159424, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5841814784, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5842470144, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5845705984, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5848941824, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5852177664, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5855413504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5858649344, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5861885184, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 12, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5862540544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5865776384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5865915648, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5869151488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5869290752, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5872526592, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5872665856, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5875901696, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5879137536, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5882373376, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5885609216, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5886264576, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5886403840, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5887059200, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5890295040, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5893530880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5896766720, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5900002560, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5903238400, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5906474240, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5909710080, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 13 } }, { "memory": 5912945920, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 13, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5916181760, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5919417600, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5922653440, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5925889280, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5929125120, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5932360960, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5935596800, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5938832640, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 13, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5942068480, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5945304320, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5948540160, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5951776000, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 13, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5955011840, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5958247680, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5961483520, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5962138880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5965374720, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5968610560, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 13, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5971846400, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 13, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5971985664, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5975221504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5975876864, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5979112704, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5982348544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 13, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5985584384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5985723648, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5988959488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5992195328, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 13, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5992334592, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 13, "layer.35.mlp.down_proj": 14 } }, { "memory": 5995570432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 5998806272, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6002042112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 13, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6005277952, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6008513792, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6011749632, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6014985472, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6015640832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6018876672, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6019532032, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6022767872, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6026003712, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6029239552, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 13, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6029894912, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6033130752, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6033786112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6034441472, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6037677312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6037816576, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6038471936, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6038611200, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6041847040, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6045082880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 13, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6048318720, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6048457984, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 13, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6051693824, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6054929664, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6058165504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6058820864, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6062056704, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6062712064, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6062851328, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6066087168, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6066226432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6069462272, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6070117632, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6073353472, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6076589312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 13, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6076728576, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6079964416, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6080619776, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6081275136, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6084510976, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6084650240, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6085305600, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6085444864, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6086100224, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6086755584, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6089991424, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6093227264, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6093882624, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6094021888, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6097257728, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6100493568, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6100632832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6103868672, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6104007936, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6104663296, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6104802560, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6108038400, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6111274240, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6111413504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6112068864, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6115304704, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6115960064, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6119195904, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 13, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6119851264, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6123087104, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 13, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6123742464, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6123881728, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6124537088, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6125192448, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6125331712, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 13, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6128567552, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6128706816, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6131942656, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6135178496, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6135317760, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6135457024, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 13, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6135596288, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6136251648, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6139487488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6140142848, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6140798208, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6144034048, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6144173312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 13, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6144828672, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6148064512, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6148719872, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6149375232, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6152611072, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6152750336, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6152889600, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6156125440, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6159361280, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6159500544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6160155904, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6160295168, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6160434432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 13, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6163670272, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6164325632, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6167561472, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6168216832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6168356096, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6168495360, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6171731200, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6174967040, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6178202880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6178342144, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6178997504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6179136768, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6179276032, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6179931392, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6180070656, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6180209920, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6180865280, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6181004544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6181659904, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6181799168, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6181938432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6182077696, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 13, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6182216960, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6182872320, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6183011584, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6183666944, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6183806208, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6184461568, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6184600832, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6185256192, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6185911552, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 13, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6186566912, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6186706176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6186845440, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6186984704, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6187640064, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6190875904, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6194111744, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6194251008, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6194390272, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6194529536, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6194668800, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 13, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6194808064, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6195463424, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6198699264, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 13, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6198838528, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6198977792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6199633152, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6200288512, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6200943872, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6204179712, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6204835072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6204974336, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6208210176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 13, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6208349440, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6208488704, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 13, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6209144064, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6209283328, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6212519168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6213174528, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6213313792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6213969152, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6217204992, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6217344256, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6217999616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6218654976, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6218794240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6218933504, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6219072768, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6222308608, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6222447872, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6225683712, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6226339072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6229574912, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6229714176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6230369536, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6231024896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6231164160, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6231819520, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6231958784, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6232098048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6232753408, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6235989248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6239225088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6242460928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6243116288, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6243771648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6243910912, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6244566272, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6247802112, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6251037952, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6251693312, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 13, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6252348672, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 13, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6253004032, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6253659392, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6256895232, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6257550592, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6260786432, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6261441792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6264677632, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6264816896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6268052736, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6268192000, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6268847360, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6272083200, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6272222464, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6275458304, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6275597568, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6278833408, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6282069248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6285305088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6288540928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6291776768, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6295012608, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6298248448, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6301484288, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6302139648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6302795008, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6303450368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6306686208, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6309922048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6313157888, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6316393728, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6319629568, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6322865408, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6326101248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6329337088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 14, "layer.35.mlp.down_proj": 14 } }, { "memory": 6332572928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 14, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6335808768, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6339044608, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6342280448, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6345516288, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6348752128, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6351987968, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6355223808, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6358459648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6361695488, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6364931328, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 14, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6368167168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6371403008, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 14, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6374638848, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 14, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6377874688, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6381110528, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6384346368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6387582208, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6388237568, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6391473408, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 14, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6394709248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6397945088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 14 } }, { "memory": 6401180928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6401836288, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6405072128, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6408307968, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 14, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6411543808, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6414779648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6418015488, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6418154752, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6418810112, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 14, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6422045952, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 14, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6425281792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6425421056, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6428656896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6431892736, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6435128576, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6435783936, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6436439296, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6439675136, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6442910976, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6446146816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6449382656, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6450038016, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6453273856, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 14, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6453413120, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6456648960, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6457304320, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6460540160, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6463776000, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6467011840, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6470247680, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6470386944, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6473622784, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6474278144, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6474417408, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6477653248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6480889088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6484124928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6484264192, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6487500032, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6490735872, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 14, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6490875136, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6491530496, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6494766336, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6498002176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6501238016, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 14, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6501377280, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6502032640, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6502688000, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6505923840, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6509159680, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6509815040, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6509954304, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6510609664, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6510748928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6511404288, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 14, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6511543552, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6514779392, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6515434752, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6515574016, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6518809856, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6518949120, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6519088384, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6522324224, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6525560064, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6525699328, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6526354688, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6527010048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6527665408, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6530901248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6531040512, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6531179776, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6531319040, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6534554880, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6535210240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6535865600, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6536004864, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 14, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6539240704, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6539379968, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6542615808, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6542755072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6543410432, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6546646272, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6547301632, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 14, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6547440896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6547580160, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6547719424, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6548374784, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6549030144, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6552265984, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6555501824, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 14, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6556157184, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6556296448, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6556435712, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6556574976, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6557230336, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6557369600, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 14, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6558024960, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 14, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6561260800, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 14, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6561916160, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 14, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6562571520, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6565807360, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 14, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6569043200, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 14, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6569698560, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6572934400, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 14, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6573073664, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6573212928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6576448768, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6577104128, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6577243392, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6577382656, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6577521920, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6578177280, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6578832640, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6579488000, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6582723840, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6582863104, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6583518464, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6584173824, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6584313088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6584968448, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6585107712, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 14, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6585246976, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6588482816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6588622080, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6588761344, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6589416704, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6590072064, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6590727424, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6590866688, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6591005952, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6591661312, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6592316672, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6592455936, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6592595200, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6592734464, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6593389824, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6593529088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6593668352, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6593807616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6594462976, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6595118336, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6598354176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6601590016, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6601729280, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6602384640, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6602523904, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6602663168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6605899008, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 14, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6606554368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6607209728, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6607865088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6608520448, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 14, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6609175808, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6609315072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 14, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6609454336, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6612690176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6615926016, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 14, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6616581376, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6616720640, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6617376000, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6618031360, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6618170624, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6618825984, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6618965248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6619620608, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6622856448, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6623511808, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6624167168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6627403008, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6628058368, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6628197632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6628336896, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6631572736, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6631712000, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6631851264, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6631990528, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6635226368, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6635881728, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6636020992, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6639256832, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6642492672, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6642631936, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6642771200, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6642910464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6646146304, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6646801664, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6650037504, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6650692864, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6650832128, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6654067968, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6657303808, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6660539648, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6663775488, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6664430848, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6667666688, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6667805952, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6671041792, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6674277632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6674416896, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6674556160, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6675211520, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6678447360, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6679102720, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6679241984, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6682477824, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6683133184, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6683272448, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6686508288, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6687163648, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6690399488, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6691054848, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6691194112, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6694429952, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6694569216, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6697805056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6698460416, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6701696256, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6702351616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6703006976, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6703662336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6706898176, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6710134016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6713369856, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6716605696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6719841536, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6723077376, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6726313216, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 15 } }, { "memory": 6729549056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 15, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6732784896, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6736020736, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6739256576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6742492416, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6745728256, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6748964096, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6749619456, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6752855296, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 15, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6756091136, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6759326976, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6762562816, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6765798656, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6769034496, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6772270336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6775506176, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6778742016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6778881280, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6782117120, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6782256384, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6782911744, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6786147584, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6789383424, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 15, "layer.35.mlp.down_proj": 16 } }, { "memory": 6792619264, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6795855104, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6796510464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6799746304, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6802982144, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6806217984, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 15, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6809453824, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6812689664, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6815925504, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6819161344, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6822397184, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 15, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6823052544, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6823707904, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 15, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6823847168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6827083008, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6830318848, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6833554688, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6836790528, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6840026368, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6843262208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6846498048, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 15, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6846637312, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6847292672, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6847431936, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6848087296, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6848742656, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6851978496, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6855214336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6855869696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6856008960, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6859244800, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6859900160, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 15, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6860039424, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6863275264, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 15, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6863930624, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6864069888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6867305728, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 15, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6870541568, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6870680832, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6871336192, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6874572032, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6875227392, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6878463232, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6881699072, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6884934912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6885590272, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6886245632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6886384896, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6889620736, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 15, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6889760000, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 15, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6892995840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6893651200, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6894306560, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6894961920, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6895617280, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6895756544, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6895895808, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6896035072, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 15, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6896174336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6896829696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6897485056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6898140416, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6901376256, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 15, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6904612096, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6905267456, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6908503296, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6908642560, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6911878400, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6915114240, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6918350080, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6918489344, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6918628608, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6918767872, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 15, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6919423232, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6922659072, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6923314432, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6923453696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6924109056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6924248320, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6924903680, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6925559040, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6928794880, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6928934144, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 15, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6929073408, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6932309248, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6932448512, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6933103872, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6933243136, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6933898496, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6934553856, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6934693120, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6935348480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6936003840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 15, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6936659200, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 15, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6936798464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6936937728, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6937076992, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 15, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6940312832, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6940968192, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6944204032, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 15, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6947439872, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6947579136, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6948234496, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6951470336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6951609600, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6952264960, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6952404224, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6953059584, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6953198848, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 15, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6953854208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6953993472, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6957229312, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6957368576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6957507840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6957647104, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6960882944, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6961538304, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6964774144, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6964913408, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6968149248, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6968804608, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6968943872, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6969599232, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6970254592, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6970393856, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6970533120, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6971188480, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6971843840, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6971983104, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6972122368, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6972777728, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6976013568, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6976152832, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6976292096, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6976431360, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6977086720, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6977225984, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6977365248, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6977504512, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6980740352, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6983976192, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6984631552, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6987867392, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6991103232, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6991758592, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6991897856, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6992037120, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6992692480, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6995928320, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 6999164160, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7002400000, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7002539264, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7003194624, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7003333888, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7003473152, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7003612416, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7006848256, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7007503616, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7008158976, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7011394816, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7011534080, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7014769920, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 15, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7014909184, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7018145024, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7018800384, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7018939648, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7019078912, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7019218176, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 15, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7019873536, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7020012800, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 15, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7020668160, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7023904000, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7024559360, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7025214720, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7025870080, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7026009344, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7026148608, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7029384448, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7030039808, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7030179072, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 15, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7030834432, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7030973696, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7031112960, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 15, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7034348800, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7037584640, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 15, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7040820480, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7044056320, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7044711680, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7047947520, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7048602880, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7051838720, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7055074560, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7055729920, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7058965760, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7059105024, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7062340864, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7062480128, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7062619392, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7065855232, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7065994496, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7066649856, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7067305216, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7070541056, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7070680320, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7073916160, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7074571520, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7077807360, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7081043200, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7084279040, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7087514880, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7090750720, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7093986560, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7094641920, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7095297280, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7095436544, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7096091904, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7099327744, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7102563584, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7103218944, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7106454784, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7109690624, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7110345984, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7113581824, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7116817664, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7120053504, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7123289344, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7126525184, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } }, { "memory": 7129761024, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16, "layer.32.self_attn.q_proj": 16, "layer.32.self_attn.k_proj": 16, "layer.32.self_attn.v_proj": 16, "layer.32.self_attn.o_proj": 16, "layer.32.mlp.gate_proj": 16, "layer.32.mlp.up_proj": 16, "layer.32.mlp.down_proj": 16, "layer.33.self_attn.q_proj": 16, "layer.33.self_attn.k_proj": 16, "layer.33.self_attn.v_proj": 16, "layer.33.self_attn.o_proj": 16, "layer.33.mlp.gate_proj": 16, "layer.33.mlp.up_proj": 16, "layer.33.mlp.down_proj": 16, "layer.34.self_attn.q_proj": 16, "layer.34.self_attn.k_proj": 16, "layer.34.self_attn.v_proj": 16, "layer.34.self_attn.o_proj": 16, "layer.34.mlp.gate_proj": 16, "layer.34.mlp.up_proj": 16, "layer.34.mlp.down_proj": 16, "layer.35.self_attn.q_proj": 16, "layer.35.self_attn.k_proj": 16, "layer.35.self_attn.v_proj": 16, "layer.35.self_attn.o_proj": 16, "layer.35.mlp.gate_proj": 16, "layer.35.mlp.up_proj": 16, "layer.35.mlp.down_proj": 16 } } ]